#!/usr/bin/env python

# encoding: utf-8

'''
 * Create File linkedin_script
 * Created by leixu on 2017/9/27
 * IDE PyCharm
'''
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
import asyncio
from luobocrawler.selenium_script.crawler import Crawler
from furl import furl
import requests
import math
from selenium import webdriver

from pymongo import MongoClient
from copy import deepcopy
import logging
import random

GET_VERIFY_CODE_TIMES = 5

_LOGGER = logging.getLogger('python-logstash-logger')
_LOGGER.setLevel(logging.INFO)


class LinkedinCrawler(Crawler):
    name_list = set()
    name_num_one_time = 2

    def __init__(self):
        super().__init__()
        self.geo_list = [
            {"台湾地区 台北市": "tw:9225"},
            {"台湾地区 新北市": "tw:9226"},
            {"台湾地区 台中市": "tw:9221"},
            {"台湾地区 桃园县": "tw:9228"},
            {"台湾地区 新竹县/市": "tw:9209"},
            {"台湾地区 高雄市": "tw:9211"},
            {"台湾地区 台南市": "tw:9223"},
            {"台湾地区 彰化县": "tw:9205"},
            {"台湾地区 嘉义县/市": "tw:9207"},
            {"台湾地区 基隆市": "tw:9213"},
            {"台湾地区 苗栗县": "tw:9216"},
            {"台湾地区 屏东县": "tw:9220"},
            {"台湾地区 宜兰县": "tw:9229"},
            {"台湾地区 南投县": "tw:9218"},
            {"台湾地区 花莲县": "tw:9210"},
            {"台湾地区 台东县": "tw:9227"},
            {"台湾地区 澎湖县": "tw:9219"},
            {"台湾地区 金门县": "tw:9214"},
            {"台湾地区 连江县": "tw:9215"}
        ]
        self.name_list = [
            ["褚"], ["李"], ["周"], ["王"], ["沈"], ["朱"], ["秦"], ["尤"], ["何"], ["施"], ["孔"], ["曹"], ["金"], ["魏"], ["陶"], ["戚"], ["喻"], ["柏"], ["水"], ["章"],
            ["潘"], ["葛"], ["奚"], ["彭"], ["郎"], ["昌"], ["苗"], ["花"], ["方"], ["俞"], ["任"], ["袁"], ["柳"], ["酆"], ["史"], ["唐"], ["廉"], ["岑"], ["薛"], ["雷"],
            ["倪"], ["滕"], ["殷"], ["郝"], ["安"], ["常"], ["傅"], ["皮"], ["卞"], ["康"], ["伍"], ["元"], ["孟"], ["平"], ["和"], ["穆"], ["尹"], ["姚"], ["邵"], ["湛"],
            ["汪"], ["祁"], ["毛"], ["禹"], ["狄"], ["米"], ["明"], ["臧"], ["伏"], ["成"], ["戴"], ["宋"], ["茅"], ["熊"], ["舒"], ["屈"], ["祝"], ["董"], ["梁"], ["杜"],
            ["阮"], ["席"], ["季"], ["麻"], ["路"], ["危"], ["江"], ["童"], ["郭"], ["梅"], ["盛"], ["林"], ["刁"], ["徐"], ["邱"], ["高"], ["夏"], ["蔡"], ["田"], ["樊"],
            ["胡"], ["霍"], ["虞"], ["支"], ["柯"], ["昝"], ["管"], ["莫"], ["房"], ["裘"], ["解"], ["宗"], ["丁"], ["宣"], ["杭"], ["洪"], ["包"], ["左"], ["石"], ["崔"],
            ["吉"], ["程"], ["嵇"], ["邢"], ["滑"], ["裴"], ["翁"], ["荀"], ["羊"], ["於"], ["惠"], ["甄"], ["麴"], ["家"], ["封"], ["芮"], ["羿"], ["靳"], ["汲"], ["邴"],
            ["糜"], ["松"], ["井"], ["段"], ["富"], ["巫"], ["焦"], ["巴"], ["弓"], ["牧"], ["隗"], ["山"], ["侯"], ["宓"], ["蓬"], ["全"], ["郗"], ["班"], ["仰"], ["秋"],
            ["仲"], ["伊"], ["仇"], ["暴"], ["甘"], ["戎"], ["祖"], ["武"], ["符"], ["景"], ["詹"], ["束"], ["幸"], ["司"], ["韶"], ["郜"], ["黎"], ["溥"], ["印"], ["宿"],
            ["白"], ["蒲"], ["邰"], ["鄂"], ["索"], ["籍"], ["卓"], ["屠"], ["蒙"], ["池"], ["胥"], ["能"], ["莘"], ["翟"], ["逄"], ["姬"], ["申"], ["扶"], ["堵"], ["冉"],
            ["宰"], ["雍"], ["璩"], ["桑"], ["桂"], ["濮"], ["牛"], ["通"], ["扈"], ["燕"], ["冀"], ["僪"], ["浦"], ["尚"], ["晏"], ["柴"], ["瞿"], ["充"], ["慕"], ["茹"],
            ["宦"], ["艾"], ["容"], ["向"], ["古"], ["易"], ["慎"], ["戈"], ["廖"], ["庾"], ["暨"], ["居"], ["衡"], ["步"], ["都"], ["耿"], ["弘"], ["匡"], ["文"], ["寇"],
            ["殳"], ["沃"], ["利"], ["蔚"], ["越"], ["夔"], ["隆"], ["晁"], ["勾"], ["敖"], ["融"], ["冷"], ["訾"], ["辛"], ["那"], ["空"], ["曾"], ["毋"], ["沙"], ["乜"],
            ["鞠"], ["巢"], ["蒯"], ["相"], ["查"], ["竺"], ["逮"], ["盍"], ["益"], ["桓"], ["公"], ["上官"], ["夏侯"], ["皇甫"], ["公羊"], ["澹台"], ["公冶"], ["宗政"], ["太叔"], ["申屠"],
            ["令狐"], ["宇文"], ["慕容"], ["司徒"], ["司空"], ["召"], ["有"], ["舜"], ["寸"], ["皇"], ["彤"], ["竭"], ["端"], ["赫"], ["甫"], ["集"], ["象"], ["翠"], ["狂"], ["辟"],
            ["典"], ["良"], ["函"], ["芒"], ["苦"], ["其"], ["京"], ["中"], ["夕"], ["之"], ["章佳"], ["那拉"], ["冠"], ["香"], ["果"], ["他塔喇"], ["太史"], ["公叔"], ["佟佳"], ["富察"],
            ["蹇"], ["多"], ["繁"], ["戊"], ["回"], ["毓"], ["靖"], ["愈"], ["牢"], ["但"], ["巧"], ["枚"], ["撒"], ["泰"], ["秘"], ["亥"], ["以"], ["壬"], ["森"], ["奕"],
            ["姒"], ["朋"], ["求"], ["羽"], ["用"], ["占"], ["真"], ["穰"], ["翦"], ["漆"], ["代"], ["旁"], ["崇"], ["告"], ["休"], ["褒"], ["皋"], ["在"], ["歧"], ["禾"],
            ["示"], ["是"], ["委"], ["嬴"], ["呼"], ["大"], ["威"], ["昂"], ["律"], ["冒"], ["保"], ["系"], ["抄"], ["定"], ["化"], ["校"], ["抗"], ["綦"], ["悟"], ["宏"],
            ["功"], ["庚"], ["敏"], ["捷"], ["拱"], ["兆"], ["丙"], ["苟"], ["卯"], ["俟"], ["友"], ["答"], ["乙"], ["允"], ["甲"], ["留"], ["尾"], ["佼"], ["玄"], ["乘"],
            ["裔"], ["延"], ["植"], ["昔"], ["侍"], ["度"], ["遇"], ["偶"], ["前"], ["由"], ["咎"], ["塞"], ["受"], ["叔"], ["夫"], ["藩"], ["邸"], ["府"], ["掌"], ["首"],
            ["焉"], ["可"], ["智"], ["悉"], ["厚"], ["仁"], ["肇"], ["合"], ["仍"], ["九"], ["衷"], ["哀"], ["刑"], ["俎"], ["仵"], ["圭"], ["夷"], ["徭"], ["汗"], ["孛"],
            ["乾"], ["帖"], ["罕"], ["洛"], ["淦"], ["洋"], ["邶"], ["郯"], ["邗"], ["邛"], ["虢"], ["隋"], ["蒿"], ["茆"], ["菅"], ["桐"], ["斛"], ["玉"], ["箕"], ["庹"],
            ["磨"], ["弭"], ["刀"], ["疏"], ["世"], ["仝"], ["同"], ["止"], ["戢"], ["睢"], ["冼"], ["肖"], ["己"], ["泣"], ["卷"], ["蹉"], ["赧"], ["浮"], ["次"], ["念"],
            ["夙"], ["斯"], ["完"], ["丹"], ["表"], ["聊"], ["源"], ["姓"], ["吾"], ["展"], ["出"], ["不"], ["才"], ["愚"], ["本"], ["性"], ["雪"], ["霜"], ["寒"], ["少"],
            ["字"], ["板"], ["斐"], ["千"], ["嘉"], ["善"], ["揭"], ["祈"], ["析"], ["赤"], ["紫"], ["青"], ["柔"], ["奇"], ["拜"], ["佛"], ["陀"], ["阿"], ["素"], ["僧"],
            ["仙"], ["宇"], ["祭"], ["酒"], ["淡"], ["塔"], ["琦"], ["始"], ["星"], ["南"], ["天"], ["接"], ["波"], ["碧"], ["速"], ["禚"], ["潮"], ["似"], ["澄"], ["潭"],
            ["謇"], ["渠"], ["奈"], ["春"], ["濯"], ["沐"], ["茂"], ["英"], ["檀"], ["藤"], ["枝"], ["生"], ["折"], ["登"], ["貊"], ["虎"], ["肥"], ["鹿"], ["雀"], ["野"],
            ["禽"], ["宜"], ["粟"], ["栗"], ["豆"], ["帛"], ["官"], ["布"], ["衣"], ["藏"], ["盈"], ["喜"], ["及"], ["普"], ["建"], ["巨"], ["望"], ["希"], ["道"], ["漫"],
            ["犁"], ["力"], ["勤"], ["革"], ["改"], ["亓"], ["睦"], ["修"], ["信"], ["北"], ["守"], ["勇"], ["尉"], ["士"], ["旅"], ["五"], ["令"], ["旗"], ["行"], ["奉"],
            ["敬"], ["恭"], ["母"], ["堂"], ["丘"], ["慈"], ["孝"], ["理"], ["卿"], ["永"], ["位"], ["依"], ["介"], ["承"], ["市"], ["所"], ["苑"], ["杞"], ["第"], ["零"],
            ["招"], ["忻"], ["六"], ["鄞"], ["候"], ["宛"], ["粘"], ["覃"], ["辜"], ["初"], ["城"], ["局"], ["台"], ["原"], ["考"], ["泉"], ["老"], ["清"], ["德"], ["卑"],
            ["曲"], ["竹"], ["百"], ["福"], ["言"], ["第五"], ["佟"], ["年"], ["笪"], ["哈"], ["墨"], ["伯"], ["佴"], ["佘"], ["牟"], ["商"], ["左丘"], ["梁丘"], ["琴"], ["亢"],
            ["微生"], ["羊舌"], ["海"], ["呼延"], ["百里"], ["鄢"], ["汝"], ["法"], ["楚"], ["谷梁"], ["宰父"], ["拓跋"], ["漆雕"], ["公西"], ["端木"], ["督"], ["仉"], ["司寇"], ["亓官"], ["逯"],
            ["逢"], ["薄"], ["稽"], ["公良"], ["光"], ["操"], ["瑞"], ["眭"], ["泥"], ["摩"], ["迮"], ["赵", "趙"], ["钱", "錢"], ["孙", "孫"], ["吴", "吳"], ["郑", "鄭"], ["冯", "馮"], ["陈", "陳"], ["卫", "衛"], ["蒋", "蔣"],
            ["韩", "韓"], ["杨", "楊"], ["许", "許"], ["吕", "呂"], ["张", "張"], ["严", "嚴"], ["华", "華"], ["姜", "薑"], ["谢", "謝"], ["邹", "鄒"], ["窦", "竇"], ["云", "雲"], ["苏", "蘇"], ["范", "範"], ["鲁", "魯"],
            ["韦", "韋"], ["马", "馬"], ["凤", "鳳"], ["鲍", "鮑"], ["费", "費"],
            ["贺", "賀"], ["汤", "湯"], ["罗", "羅"], ["毕", "畢"], ["邬", "鄔"], ["乐", "樂"], ["于", "於"], ["时", "時"], ["齐", "齊"], ["余", "餘"], ["卜", "蔔"], ["顾", "顧"], ["黄", "黃"], ["萧", "蕭"], ["贝", "貝"],
            ["计", "計"], ["谈", "談"], ["庞", "龐"], ["纪", "紀"], ["项", "項"],
            ["蓝", "藍"], ["闵", "閔"], ["强", "強"], ["贾", "賈"], ["娄", "婁"], ["颜", "顏"], ["锺", "鍾"], ["骆", "駱"], ["凌", "淩"], ["万", "萬"], ["卢", "盧"], ["经", "經"], ["缪", "繆"], ["干", "幹"], ["应", "應"],
            ["贲", "賁"], ["邓", "鄧"], ["郁", "鬱"], ["单", "單"], ["诸", "諸"],
            ["钮", "鈕"], ["龚", "龔"], ["陆", "陸"], ["荣", "榮"], ["储", "儲"], ["乌", "烏"], ["谷", "穀"], ["车", "車"], ["宫", "宮"], ["宁", "寧"], ["栾", "欒"], ["钭", "鈄"], ["历", "曆"], ["刘", "劉"], ["龙", "龍"],
            ["叶", "葉"], ["蓟", "薊"], ["怀", "懷"], ["从", "從"], ["咸", "鹹"],
            ["赖", "賴"], ["蔺", "藺"], ["乔", "喬"], ["阳", "陽"], ["苍", "蒼"], ["双", "雙"], ["闻", "聞"], ["党", "黨"], ["谭", "譚"], ["贡", "貢"], ["劳", "勞"], ["郦", "酈"], ["却", "卻"], ["寿", "壽"], ["边", "邊"],
            ["农", "農"], ["温", "溫"], ["别", "別"], ["庄", "莊"], ["阎", "閻"],
            ["连", "連"], ["习", "習"], ["鱼", "魚"], ["终", "終"], ["满", "滿"], ["国", "國"], ["广", "廣"], ["禄", "祿"], ["阙", "闕"], ["东", "東"], ["欧", "歐"], ["师", "師"], ["巩", "鞏"], ["厍", "厙"], ["聂", "聶"],
            ["阚", "闞"], ["简", "簡"], ["饶", "饒"], ["养", "養"], ["须", "須"],
            ["丰", "豐"], ["关", "關"], ["后", "後"], ["荆", "荊"], ["红", "紅"], ["游", "遊"], ["权", "權"], ["万俟", "萬俟"], ["司马", "司馬"], ["欧阳", "歐陽"], ["诸葛", "諸葛"], ["闻人", "聞人"], ["东方", "東方"], ["赫连", "赫連"],
            ["尉迟", "尉遲"], ["濮阳", "濮陽"], ["淳于", "淳於"], ["单于", "單于"], ["公孙", "公孫"], ["仲孙", "仲孫"],
            ["轩辕", "軒轅"], ["钟离", "鐘離"], ["长孙", "長孫"], ["叶赫那拉", "葉赫那拉"], ["丛", "叢"], ["岳", "嶽"], ["贰", "貳"], ["侨", "僑"], ["实", "實"], ["宾", "賓"], ["依尔根觉罗", "依爾根覺羅"], ["依尔觉罗", "依爾覺羅"], ["萨嘛喇", "薩嘛喇"],
            ["赫舍里", "赫舍裡"], ["额尔德特", "額爾德特"], ["萨克达", "薩克達"], ["钮祜禄", "鈕祜祿"], ["喜塔腊", "喜塔臘"], ["讷殷富察", "訥殷富察"], ["叶赫那兰", "葉赫那蘭"],
            ["库雅喇", "庫雅喇"], ["瓜尔佳", "瓜爾佳"], ["舒穆禄", "舒穆祿"], ["爱新觉罗", "愛新覺羅"], ["索绰络", "索綽絡"], ["纳喇", "納喇"], ["乌雅", "烏雅"], ["范姜", "范薑"], ["碧鲁", "碧魯"], ["张廖", "張廖"], ["张简", "張簡"], ["图门", "圖門"],
            ["乌孙", "烏孫"], ["完颜", "完顏"], ["马佳", "馬佳"], ["费莫", "費莫"], ["称", "稱"], ["诺", "諾"], ["来", "來"], ["朴", "樸"],
            ["税", "稅"], ["荤", "葷"], ["绪", "緒"], ["硕", "碩"], ["买", "買"], ["绍", "紹"], ["斋", "齋"], ["释", "釋"], ["闾", "閭"], ["贵", "貴"], ["贯", "貫"], ["栋", "棟"], ["谏", "諫"], ["锐", "銳"], ["闳", "閎"],
            ["钊", "釗"], ["频", "頻"], ["莱", "萊"], ["么", "麼"], ["祢", "禰"],
            ["务", "務"], ["丑", "醜"], ["畅", "暢"], ["随", "隨"], ["类", "類"], ["环", "環"], ["矫", "矯"], ["赛", "賽"], ["旷", "曠"], ["敛", "斂"], ["泷", "瀧"], ["袭", "襲"], ["衅", "釁"], ["圣", "聖"], ["御", "禦"],
            ["仆", "僕"], ["镇", "鎮"], ["员", "員"], ["戏", "戲"], ["尔", "爾"],
            ["凭", "憑"], ["进", "進"], ["笃", "篤"], ["业", "業"], ["资", "資"], ["蛮", "蠻"], ["郸", "鄲"], ["剑", "劍"], ["苌", "萇"], ["树", "樹"], ["锁", "鎖"], ["钟", "鐘"], ["机", "機"], ["盘", "盤"], ["铎", "鐸"],
            ["线", "線"], ["针", "針"], ["绳", "繩"], ["蒉", "蕢"], ["瓮", "甕"],
            ["牵", "牽"], ["浑", "渾"], ["恽", "惲"], ["势", "勢"], ["蚁", "蟻"], ["种", "種"], ["涂", "塗"], ["潜", "潛"], ["脱", "脫"], ["谬", "謬"], ["顿", "頓"], ["说", "說"], ["错", "錯"], ["寻", "尋"], ["户", "戶"],
            ["闭", "閉"], ["无", "無"], ["书", "書"], ["学", "學"], ["烟", "煙"],
            ["桥", "橋"], ["独", "獨"], ["诗", "詩"], ["扬", "揚"], ["刚", "剛"], ["弥", "彌"], ["长", "長"], ["隐", "隱"], ["隽", "雋"], ["闪", "閃"], ["腾", "騰"], ["镜", "鏡"], ["纵", "縱"], ["风", "風"], ["兰", "蘭"],
            ["检", "檢"], ["驹", "駒"], ["骑", "騎"], ["飞", "飛"], ["节", "節"],
            ["鲜", "鮮"], ["宝", "寶"], ["钞", "鈔"], ["银", "銀"], ["门", "門"], ["庆", "慶"], ["营", "營"], ["载", "載"], ["声", "聲"], ["贸", "貿"], ["兴", "興"], ["闽", "閩"], ["坚", "堅"], ["汉", "漢"], ["练", "練"],
            ["将", "將"], ["军", "軍"], ["仪", "儀"], ["义", "義"], ["礼", "禮"],
            ["伦", "倫"], ["问", "問"], ["辉", "輝"], ["让", "讓"], ["尧", "堯"], ["犹", "猶"], ["剧", "劇"], ["谌", "諶"], ["续", "續"], ["达", "達"], ["战", "戰"], ["迟", "遲"], ["励", "勵"], ["萨", "薩"], ["邝", "鄺"],
            ["楼", "樓"], ["区", "區"], ["妫", "媯"], ["纳", "納"], ["过", "過"],
            ["麦", "麥"], ["爱", "愛"], ["谯", "譙"], ["南宫", "南宮"], ["赏", "賞"], ["西门", "西門"], ["东门", "東門"], ["况", "況"], ["缑", "緱"], ["帅", "帥"], ["归", "歸"], ["南门", "南門"], ["东郭", "東郭"], ["钦", "欽"],
            ["闫", "閆"], ["晋", "晉"], ["夹谷", "夾穀"], ["壤驷", "壤駟"], ["乐正", "樂正"], ["巫马", "巫馬"],
            ["颛孙", "顓孫"], ["子车", "子車"], ["鲜于", "鮮於"], ["锺离", "鍾離"], ["盖", "蓋"], ["库", "庫"], ["郏", "郟"], ["阴", "陰"], ["厉", "厲"], ["闾丘", "閭丘"], ["段干", "段幹"], ["开", "開"], ["运", "運"], ["伟", "偉"],
            ["铁", "鐵"], ["zhao"], ["qian"], ["sun"], ["li"], ["zhou"],
            ["wu"], ["zheng"], ["wang"], ["feng"], ["chen"], ["chu"], ["wei"], ["jiang"], ["shen"], ["han"], ["yang"], ["zhu"], ["qin"], ["you"], ["xu"], ["he"], ["lv"], ["shi"], ["zhang"], ["kong"],
            ["cao"], ["yan"], ["hua"], ["jin"], ["tao"], ["qi"], ["xie"], ["zou"], ["yu"], ["bai"], ["shui"], ["dou"], ["yun"], ["su"], ["pan"], ["ge"], ["xi"], ["fan"], ["peng"], ["lang"],
            ["lu"], ["chang"], ["ma"], ["miao"], ["fang"], ["ren"], ["yuan"], ["liu"], ["bao"], ["tang"], ["fei"], ["lian"], ["cen"], ["xue"], ["lei"], ["teng"], ["yin"], ["luo"], ["bi"], ["hao"],
            ["an"], ["le"], ["fu"], ["pi"], ["bian"], ["kang"], ["bo"], ["gu"], ["meng"], ["ping"], ["huang"], ["mu"], ["xiao"], ["yao"], ["shao"], ["zhan"], ["mao"], ["di"], ["mi"], ["bei"],
            ["ming"], ["zang"], ["ji"], ["cheng"], ["dai"], ["tan"], ["song"], ["pang"], ["xiong"], ["shu"], ["qu"], ["xiang"], ["dong"], ["liang"], ["ruan"], ["lan"], ["min"], ["qiang"], ["jia"],
            ["lou"],
            ["tong"], ["guo"], ["mei"], ["sheng"], ["lin"], ["diao"], ["zhong"], ["qiu"], ["gao"], ["xia"], ["cai"], ["tian"], ["hu"], ["ling"], ["huo"], ["wan"], ["zhi"], ["ke"], ["zan"], ["guan"],
            ["mo"], ["jing"], ["gan"], ["jie"], ["ying"], ["zong"], ["ding"], ["xuan"], ["ben"], ["deng"], ["dan"], ["hang"], ["hong"], ["zuo"], ["cui"], ["niu"], ["gong"], ["xing"], ["pei"],
            ["rong"],
            ["weng"], ["xun"], ["hui"], ["zhen"], ["rui"], ["yi"], ["bing"], ["jiao"], ["ba"], ["kui"], ["shan"], ["che"], ["hou"], ["quan"], ["ban"], ["ning"], ["chou"], ["luan"], ["tou"], ["zu"],
            ["long"], ["ye"], ["si"], ["pu"], ["huai"], ["tai"], ["cong"], ["suo"], ["xian"], ["lai"], ["zhuo"], ["tu"], ["chi"], ["qiao"], ["neng"], ["cang"], ["shuang"], ["wen"], ["xin"], ["dang"],
            ["lao"], ["ran"], ["zai"], ["yong"], ["que"], ["sang"], ["gui"], ["shou"], ["ju"], ["shang"], ["nong"], ["bie"], ["zhuang"], ["chai"], ["chong"], ["ru"], ["huan"], ["ai"], ["liao"],
            ["heng"],
            ["bu"], ["geng"], ["man"], ["kuang"], ["kou"], ["guang"], ["ou"], ["wo"], ["yue"], ["she"], ["nie"], ["chao"], ["gou"], ["ao"], ["leng"], ["zi"], ["kan"], ["na"], ["jian"], ["rao"],
            ["zeng"], ["sha"], ["kuai"], ["cha"], ["du"], ["shun"], ["cun"], ["dian"], ["mang"], ["ku"], ["bin"], ["nuo"], ["hun"], ["shuo"], ["mai"], ["sa"], ["hai"], ["sen"], ["zhai"], ["rang"],
            ["xiu"], ["pin"], ["da"], ["ang"], ["me"], ["sui"], ["sai"], ["jiu"], ["reng"], ["tie"], ["qiong"], ["duo"], ["tuo"], ["dao"], ["juan"], ["miu"], ["cuo"], ["nan"], ["ci"], ["nian"],
            ["biao"], ["qing"], ["rou"], ["gang"], ["fo"], ["a"], ["seng"], ["ta"], ["nai"], ["chun"], ["zhe"], ["men"], ["gai"], ["jun"], ["lun"], ["kao"], ["de"], ["di wu"], ["ha"], ["mou"],
            ["shuai"], ["fa"], ["kai"], ["ni"], ["ze"], ["pai"], ["pao"], ["pien"], ["tsao"], ["tsen"], ["ko"], ["ho"], ["chiang"], ["chin"], ["kung"], ["lien"], ["lo"], ["chien"], ["jen"], ["shih"],
            ["hsi"], ["hsiao"], ["hsieh"], ["hsu"], ["hsueh"], ["yen"], ["yueh"], ["tsou"], ["on"], ["pak"], ["pau"], ["pat"], ["mary"], ["patricia"], ["linda"], ["barbara"], ["elizabeth"],
            ["jennifer"], ["maria"], ["susan"],
            ["margaret"], ["dorothy"], ["lisa"], ["nancy"], ["karen"], ["betty"], ["helen"], ["sandra"], ["donna"], ["carol"], ["ruth"], ["sharon"], ["michelle"], ["laura"], ["sarah"], ["kimberly"],
            ["deborah"], ["jessica"], ["shirley"], ["cynthia"],
            ["angela"], ["melissa"], ["brenda"], ["amy"], ["anna"], ["rebecca"], ["virginia"], ["kathleen"], ["pamela"], ["martha"], ["debra"], ["amanda"], ["stephanie"], ["carolyn"], ["christine"],
            ["marie"], ["janet"], ["catherine"], ["frances"], ["ann"],
            ["joyce"], ["diane"], ["alice"], ["julie"], ["heather"], ["teresa"], ["doris"], ["gloria"], ["evelyn"], ["jean"], ["cheryl"], ["mildred"], ["katherine"], ["joan"], ["ashley"], ["judith"],
            ["rose"], ["janice"], ["kelly"], ["nicole"],
            ["judy"], ["christina"], ["kathy"], ["theresa"], ["beverly"], ["denise"], ["tammy"], ["irene"], ["jane"], ["lori"], ["rachel"], ["marilyn"], ["andrea"], ["kathryn"], ["louise"], ["sara"],
            ["anne"], ["jacqueline"], ["wanda"], ["bonnie"],
            ["julia"], ["ruby"], ["lois"], ["tina"], ["phyllis"], ["norma"], ["paula"], ["diana"], ["annie"], ["lillian"], ["emily"], ["robin"], ["peggy"], ["crystal"], ["gladys"], ["rita"], ["dawn"],
            ["connie"], ["florence"], ["tracy"],
            ["edna"], ["tiffany"], ["carmen"], ["rosa"], ["cindy"], ["grace"], ["wendy"], ["victoria"], ["edith"], ["kim"], ["sherry"], ["sylvia"], ["josephine"], ["thelma"], ["shannon"], ["sheila"],
            ["ethel"], ["ellen"], ["elaine"], ["marjorie"],
            ["carrie"], ["charlotte"], ["monica"], ["esther"], ["pauline"], ["emma"], ["juanita"], ["anita"], ["rhonda"], ["hazel"], ["amber"], ["eva"], ["debbie"], ["april"], ["leslie"], ["clara"],
            ["lucille"], ["jamie"], ["joanne"], ["eleanor"],
            ["valerie"], ["danielle"], ["megan"], ["alicia"], ["suzanne"], ["michele"], ["gail"], ["bertha"], ["darlene"], ["veronica"], ["jill"], ["erin"], ["geraldine"], ["lauren"], ["cathy"],
            ["joann"], ["lorraine"], ["lynn"], ["sally"], ["regina"],
            ["erica"], ["beatrice"], ["dolores"], ["bernice"], ["audrey"], ["yvonne"], ["annette"], ["june"], ["samantha"], ["marion"], ["dana"], ["stacy"], ["ana"], ["renee"], ["ida"], ["vivian"],
            ["roberta"], ["holly"], ["brittany"], ["melanie"],
            ["loretta"], ["yolanda"], ["jeanette"], ["laurie"], ["katie"], ["kristen"], ["vanessa"], ["alma"], ["sue"], ["elsie"], ["beth"], ["jeanne"], ["vicki"], ["carla"], ["tara"], ["rosemary"],
            ["eileen"], ["terri"], ["gertrude"], ["lucy"],
            ["tonya"], ["ella"], ["stacey"], ["wilma"], ["gina"], ["kristin"], ["jessie"], ["natalie"], ["agnes"], ["vera"], ["willie"], ["charlene"], ["bessie"], ["delores"], ["melinda"], ["pearl"],
            ["arlene"], ["maureen"], ["colleen"], ["allison"],
            ["tamara"], ["joy"], ["georgia"], ["constance"], ["lillie"], ["claudia"], ["jackie"], ["marcia"], ["tanya"], ["nellie"], ["minnie"], ["marlene"], ["heidi"], ["glenda"], ["lydia"],
            ["viola"], ["courtney"], ["marian"], ["stella"], ["caroline"],
            ["dora"], ["jo"], ["vickie"], ["mattie"], ["terry"], ["maxine"], ["irma"], ["mabel"], ["marsha"], ["myrtle"], ["lena"], ["christy"], ["deanna"], ["patsy"], ["hilda"], ["gwendolyn"],
            ["jennie"], ["nora"], ["margie"], ["nina"],
            ["cassandra"], ["leah"], ["penny"], ["kay"], ["priscilla"], ["naomi"], ["carole"], ["brandy"], ["olga"], ["billie"], ["dianne"], ["tracey"], ["leona"], ["jenny"], ["felicia"], ["sonia"],
            ["miriam"], ["velma"], ["becky"], ["bobbie"],
            ["violet"], ["kristina"], ["toni"], ["misty"], ["mae"], ["shelly"], ["daisy"], ["ramona"], ["sherri"], ["erika"], ["katrina"], ["claire"], ["lindsey"], ["lindsay"], ["geneva"],
            ["guadalupe"], ["belinda"], ["margarita"], ["sheryl"], ["cora"],
            ["faye"], ["ada"], ["natasha"], ["sabrina"], ["isabel"], ["marguerite"], ["hattie"], ["harriet"], ["molly"], ["cecilia"], ["kristi"], ["brandi"], ["blanche"], ["sandy"], ["rosie"],
            ["joanna"], ["iris"], ["eunice"], ["angie"], ["inez"],
            ["lynda"], ["madeline"], ["amelia"], ["alberta"], ["genevieve"], ["monique"], ["jodi"], ["janie"], ["maggie"], ["kayla"], ["sonya"], ["jan"], ["lee"], ["kristine"], ["candace"],
            ["fannie"], ["maryann"], ["opal"], ["alison"], ["yvette"],
            ["melody"], ["luz"], ["susie"], ["olivia"], ["flora"], ["shelley"], ["kristy"], ["mamie"], ["lula"], ["lola"], ["verna"], ["beulah"], ["antoinette"], ["candice"], ["juana"], ["jeannette"],
            ["pam"], ["kelli"], ["hannah"], ["whitney"],
            ["bridget"], ["karla"], ["celia"], ["latoya"], ["patty"], ["shelia"], ["gayle"], ["della"], ["vicky"], ["lynne"], ["sheri"], ["marianne"], ["kara"], ["jacquelyn"], ["erma"], ["blanca"],
            ["myra"], ["leticia"], ["krista"], ["roxanne"],
            ["angelica"], ["johnnie"], ["robyn"], ["francis"], ["adrienne"], ["rosalie"], ["alexandra"], ["brooke"], ["bethany"], ["sadie"], ["bernadette"], ["traci"], ["jody"], ["kendra"],
            ["jasmine"], ["nichole"], ["rachael"], ["chelsea"], ["mable"], ["ernestine"],
            ["muriel"], ["marcella"], ["elena"], ["krystal"], ["angelina"], ["nadine"], ["kari"], ["estelle"], ["dianna"], ["paulette"], ["lora"], ["mona"], ["doreen"], ["rosemarie"], ["angel"],
            ["desiree"], ["antonia"], ["hope"], ["ginger"], ["janis"],
            ["betsy"], ["christie"], ["freda"], ["mercedes"], ["meredith"], ["lynette"], ["teri"], ["cristina"], ["eula"], ["leigh"], ["meghan"], ["sophia"], ["eloise"], ["rochelle"], ["gretchen"],
            ["cecelia"], ["raquel"], ["henrietta"], ["alyssa"], ["jana"],
            ["kelley"], ["gwen"], ["kerry"], ["jenna"], ["tricia"], ["laverne"], ["olive"], ["alexis"], ["tasha"], ["silvia"], ["elvira"], ["casey"], ["delia"], ["sophie"], ["kate"], ["patti"],
            ["lorena"], ["kellie"], ["sonja"], ["lila"],
            ["lana"], ["darla"], ["may"], ["mindy"], ["essie"], ["mandy"], ["lorene"], ["elsa"], ["josefina"], ["jeannie"], ["miranda"], ["dixie"], ["lucia"], ["marta"], ["faith"], ["lela"],
            ["johanna"], ["shari"], ["camille"], ["tami"],
            ["shawna"], ["elisa"], ["ebony"], ["melba"], ["ora"], ["nettie"], ["tabitha"], ["ollie"], ["jaime"], ["winifred"], ["kristie"], ["james"], ["john"], ["robert"], ["michael"], ["william"],
            ["david"], ["richard"], ["charles"], ["joseph"],
            ["thomas"], ["christopher"], ["daniel"], ["paul"], ["mark"], ["donald"], ["george"], ["kenneth"], ["steven"], ["edward"], ["brian"], ["ronald"], ["anthony"], ["kevin"], ["jason"],
            ["matthew"], ["gary"], ["timothy"], ["jose"], ["larry"],
            ["jeffrey"], ["frank"], ["scott"], ["eric"], ["stephen"], ["andrew"], ["raymond"], ["gregory"], ["joshua"], ["jerry"], ["dennis"], ["walter"], ["patrick"], ["peter"], ["harold"],
            ["douglas"], ["henry"], ["carl"], ["arthur"], ["ryan"],
            ["roger"], ["joe"], ["jack"], ["albert"], ["jonathan"], ["justin"], ["gerald"], ["keith"], ["samuel"], ["ralph"], ["lawrence"], ["nicholas"], ["roy"], ["benjamin"], ["bruce"], ["brandon"],
            ["adam"], ["harry"], ["fred"], ["wayne"],
            ["billy"], ["steve"], ["louis"], ["jeremy"], ["aaron"], ["randy"], ["howard"], ["eugene"], ["carlos"], ["russell"], ["bobby"], ["victor"], ["martin"], ["ernest"], ["phillip"], ["todd"],
            ["jesse"], ["craig"], ["alan"], ["shawn"],
            ["clarence"], ["sean"], ["philip"], ["chris"], ["johnny"], ["earl"], ["jimmy"], ["antonio"], ["danny"], ["bryan"], ["tony"], ["luis"], ["mike"], ["stanley"], ["leonard"], ["nathan"],
            ["dale"], ["manuel"], ["rodney"], ["curtis"],
            ["norman"], ["allen"], ["marvin"], ["vincent"], ["glenn"], ["jeffery"], ["travis"], ["jeff"], ["chad"], ["jacob"], ["melvin"], ["alfred"], ["kyle"], ["bradley"], ["jesus"], ["herbert"],
            ["frederick"], ["ray"], ["joel"], ["edwin"],
            ["don"], ["eddie"], ["ricky"], ["troy"], ["randall"], ["barry"], ["alexander"], ["bernard"], ["mario"], ["leroy"], ["francisco"], ["marcus"], ["micheal"], ["theodore"], ["clifford"],
            ["miguel"], ["oscar"], ["jay"], ["jim"], ["tom"],
            ["calvin"], ["alex"], ["jon"], ["ronnie"], ["bill"], ["lloyd"], ["tommy"], ["leon"], ["derek"], ["warren"], ["darrell"], ["jerome"], ["floyd"], ["leo"], ["alvin"], ["tim"], ["wesley"],
            ["gordon"], ["dean"], ["greg"],
            ["jorge"], ["dustin"], ["pedro"], ["derrick"], ["lewis"], ["zachary"], ["corey"], ["herman"], ["maurice"], ["vernon"], ["roberto"], ["clyde"], ["glen"], ["hector"], ["shane"], ["ricardo"],
            ["sam"], ["rick"], ["lester"], ["brent"],
            ["ramon"], ["charlie"], ["tyler"], ["gilbert"], ["gene"], ["marc"], ["reginald"], ["ruben"], ["brett"], ["nathaniel"], ["rafael"], ["edgar"], ["milton"], ["raul"], ["chester"], ["cecil"],
            ["duane"], ["franklin"], ["andre"], ["elmer"],
            ["brad"], ["gabriel"], ["ron"], ["mitchell"], ["roland"], ["arnold"], ["harvey"], ["jared"], ["adrian"], ["karl"], ["cory"], ["claude"], ["erik"], ["darryl"], ["neil"], ["christian"],
            ["javier"], ["fernando"], ["clinton"], ["ted"],
            ["mathew"], ["tyrone"], ["darren"], ["lonnie"], ["lance"], ["cody"], ["julio"], ["kurt"], ["allan"], ["nelson"], ["guy"], ["clayton"], ["hugh"], ["max"], ["dwayne"], ["dwight"],
            ["armando"], ["felix"], ["jimmie"], ["everett"],
            ["jordan"], ["ian"], ["wallace"], ["ken"], ["bob"], ["alfredo"], ["alberto"], ["dave"], ["ivan"], ["sidney"], ["byron"], ["julian"], ["isaac"], ["morris"], ["clifton"], ["willard"],
            ["daryl"], ["ross"], ["virgil"], ["andy"],
            ["marshall"], ["salvador"], ["perry"], ["kirk"], ["sergio"], ["seth"], ["kent"], ["terrance"], ["rene"], ["eduardo"], ["terrence"], ["enrique"], ["freddie"], ["wade"], ["austin"],
            ["stuart"], ["fredrick"], ["arturo"], ["alejandro"], ["joey"],
            ["nick"], ["luther"], ["wendell"], ["jeremiah"], ["evan"], ["julius"], ["donnie"], ["otis"], ["trevor"], ["oliver"], ["luke"], ["homer"], ["gerard"], ["doug"], ["kenny"], ["hubert"],
            ["angelo"], ["shaun"], ["lyle"], ["matt"],
            ["alfonso"], ["orlando"], ["rex"], ["carlton"], ["ernesto"], ["cameron"], ["neal"], ["pablo"], ["lorenzo"], ["omar"], ["wilbur"], ["blake"], ["grant"], ["horace"], ["roderick"],
            ["abraham"], ["willis"], ["rickey"], ["ira"], ["andres"],
            ["cesar"], ["johnathan"], ["malcolm"], ["rudolph"], ["damon"], ["kelvin"], ["rudy"], ["preston"], ["alton"], ["archie"], ["marco"], ["wm"], ["pete"], ["randolph"], ["garry"], ["geoffrey"],
            ["jonathon"], ["felipe"], ["bennie"], ["gerardo"],
            ["ed"], ["dominic"], ["loren"], ["delbert"], ["colin"], ["guillermo"], ["earnest"], ["lucas"], ["benny"], ["noel"], ["spencer"], ["rodolfo"], ["myron"], ["edmund"], ["garrett"],
            ["salvatore"], ["cedric"], ["lowell"], ["gregg"], ["sherman"],
            ["wilson"], ["devin"], ["sylvester"], ["roosevelt"], ["israel"], ["jermaine"], ["forrest"], ["wilbert"], ["leland"], ["simon"], ["clark"], ["irving"], ["carroll"], ["bryant"], ["owen"],
            ["rufus"], ["woodrow"], ["sammy"], ["kristopher"], ["mack"],
            ["levi"], ["marcos"], ["gustavo"], ["jake"], ["lionel"], ["marty"], ["taylor"], ["ellis"], ["dallas"], ["gilberto"], ["clint"], ["nicolas"], ["laurence"], ["ismael"], ["orville"],
            ["drew"], ["ervin"], ["dewey"], ["al"], ["wilfred"],
            ["josh"], ["hugo"], ["ignacio"], ["caleb"], ["tomas"], ["sheldon"], ["erick"], ["frankie"], ["stewart"], ["doyle"], ["darrel"], ["rogelio"], ["terence"], ["santiago"], ["alonzo"],
            ["elias"], ["bert"], ["elbert"], ["ramiro"], ["conrad"],
            ["noah"], ["grady"], ["phil"], ["cornelius"], ["lamar"], ["rolando"], ["clay"], ["percy"], ["dexter"], ["bradford"], ["merle"], ["darin"], ["amos"], ["terrell"], ["moses"], ["irvin"],
            ["saul"], ["roman"], ["darnell"], ["randal"],
            ["tommie"], ["timmy"], ["darrin"], ["winston"], ["brendan"], ["toby"], ["van"], ["abel"], ["dominick"], ["boyd"], ["emilio"], ["elijah"], ["cary"], ["domingo"], ["santos"], ["aubrey"],
            ["emmett"], ["marlon"], ["emanuel"], ["jerald"], ["edmond"]
        ]
        self.name_list.reverse()
        self.college_list = [
            {"臺灣大學": "17469"},
            {"政治大學": "17443"},
            {"交通大學": "17445"}, {"臺灣科技大學": "17471"}, {"清華大學": "17472"},
            {"輔仁大學": "17424"}, {"淡江大學": "17490"}, {"中央大學": "17440"}, {"東吳大學": "17492"}, {"中山大學": "17461"},
            {"成功大學": "17442"}, {"中原大學": "17422"}, {"中國文化大學": "17416"}, {"逢甲大學": "17484"}, {"中興大學": "17467"},
            {"東海大學": "17492"}, {"臺北科技大學": "17448"}, {"銘傳大學": "17436"}, {"元智大學": "17495"}, {"臺北大學": "17463"},
            {"世新大學": "17477"}, {"實踐大學": "17481"}, {"臺灣海洋大學": "17468"}, {"東華大學": "17449"}, {"雲林科技大學": "17478"},
            {"靜宜大學": "17480"}, {"長庚大學": "17408"}, {"陽明大學": "17477"}, {"大同大學": "17491"}, {"高雄應用科技大學": "17457"},
            {"義守大學": "17428"}, {"臺北商業技術學院": "38781"}, {"中華大學": "17420"}, {"高雄第一科技大學": "17454"}, {"高雄大學": "17474"},
            {"亞洲大學": "17407"}, {"長榮大學": "17409"}, {"台南大學": "17475"}, {"慈濟大學": "17493"}, {"開南大學": "17429"},
            {"中正大學": "17447"}, {"宜蘭大學": "17453"}, {"嘉義大學": "17446"}
        ]
        self.client = MongoClient('192.168.2.24').Linkedin
        self.top_node = SearchNode()

    def init_env(self):
        self.ua = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:57.0) Gecko/20100101 Firefox/57.0"
        self.proxy = "192.168.2.24:20105"
        # 指定selenium driver 的类型，默认为chrome
        self.driver_type = "chrome"
        # selenium drvier config,默认采用默认配置
        self.driver_config = custom_chrome_config(is_headless=False, _proxy=self.proxy, ua=self.ua)
        # 默认配置配置的简单配置项目
        self.ref_config = {}

    async def check_and_relogin(self, driver, username, password):
        _LOGGER.info("开始检查url是否合规，主要检查是否进入了ip的检查导致的相关的账户验证")
        driver.get("https://www.linkedin.com/recruiter/smartsearch")
        await asyncio.sleep(2)
        c_url = driver.current_url
        if "www.linkedin.com/uas/login-cap" in c_url:
            driver.find_element_by_xpath("//input[@id='session_key-login']").send_keys(username)
            driver.find_element_by_xpath("//input[@id='session_password-login']").send_keys(password, Keys.RETURN)
            await asyncio.sleep(5)
            if "consumer-email-challenge" in driver.current_url:
                _LOGGER.info("当前账户需求验证", extra={
                    "当前username": "username"
                })

                async def get_verify_code(i_times=0):
                    r = requests.post(f'http://{API_SERVER}:{API_PORT}/operation/quick/email/get_linkedin_verify_code', data={'email_username': username})
                    if r.text:
                        driver.find_element_by_xpath("//input[@id='verification-code']").send_keys(r.text, Keys.RETURN)
                    else:
                        i_times = i_times + 1
                        if i_times == GET_VERIFY_CODE_TIMES:
                            raise Exception(f"Try {times} times to get verify code ,failed.")
                        await asyncio.sleep(5)
                        await get_verify_code(i_times=i_times)

                await get_verify_code()
            driver.get("https://www.linkedin.com/recruiter/smartsearch")
            try:
                element = WebDriverWait(driver, 20).until(
                    EC.presence_of_element_located((By.XPATH, "//button[@data-tracking-control-name]")))
            except:
                return None

    async def rule_1(self):
        pass

    # 该条规则主要针对的是以地区为主要的搜索框
    async def rule_2(self):
        username = "huayi123.feng@outlook.com"
        password = "!qaz2wsx3edc"
        await self.login(username, password)
        await self.fetch_info_data(self.driver, None, username, password)

    async def login(self, username, password):
        driver = self.driver
        driver.implicitly_wait(5)
        driver.set_page_load_timeout(20)
        # 检查url是否符合要求
        await self.check_and_relogin(driver, username, password)

        async def search_key_data():

            async def input_key(_key):
                driver.find_element_by_xpath("//button[@data-tracking-control-name]").click()
                await asyncio.sleep(1)
                driver.find_element_by_xpath("//div/span/span/input[@role='combobox']").send_keys(f"{_key}", Keys.RETURN)

            # 输入 搜索关键字

            await input_key("台湾")

        try:
            element = WebDriverWait(driver, 20).until(
                EC.presence_of_element_located((By.XPATH, "//div[@id='top-bar']/h2[@id]/span")))
        except Exception as e:
            return

    async def fetch_info_data(self, driver: webdriver, parent_node, username, password):
        _LOGGER.info("开启进行info的抓取")
        totalcount = 1
        url = driver.current_url

        if "searchCacheKey" in url:
            _LOGGER.info("进入抓取逻辑")

            # load url
            _ = furl(url)

            current_node = SearchNode()
            current_node.save_or_update_search_info(deepcopy(_.args))

            if parent_node:
                current_node.set_parent_node(parent_node)

            import requests
            proxies = {'all_proxy': f'socks5://{self.proxy}'}
            with requests.Session() as session:
                session.headers.update({"user-agent": self.ua,
                                        "accept-language": "zh-TW,zh;q=0.8,en-US;q=0.5,en;q=0.3",
                                        "accept-encoding": "gzip, deflate, br",
                                        "x-requested-with": "XMLHttpRequest",
                                        "Host": "www.linkedin.com",
                                        "Accept": "application/json, text/javascript, */*; q=0.01",
                                        "Connections": "keep-alive"})
                session.proxies.update(proxies)

                start_index = 0

                # 设置结果请求cookie
                async def set_cookie():
                    # set request cookies
                    cookies = driver.get_cookies()
                    for cookie in cookies:
                        session.cookies.set(cookie['name'], cookie['value'])

                # 生成search的
                def gen_search_url(_my_url, _type=0, _parent_node=None, **kwargs):
                    url_handler = URLHandler(deepcopy(_my_url))
                    if _type == 0:
                        url_handler.add_geo_scope("tw:0")

                    elif _type == 1:
                        urls_list = []
                        for last_names in self.name_list:
                            _copy_url_handler = deepcopy(url_handler)
                            for name_item in last_names:
                                _copy_url_handler.add_last_name(name_item)
                            _current_node = SearchNode()
                            _current_node.save_or_update_search_info(deepcopy(_.args))
                            if _parent_node:
                                _current_node.set_parent_node(parent_node)
                            urls_list.append(_copy_url_handler.get_my_url())
                        return urls_list

                    elif _type == 2:
                        urls_list = []
                        for i in self.geo_list[1:]:
                            for key, value in i.items():
                                _copy_url_handler = deepcopy(url_handler)
                                _copy_url_handler.clear_geo_scope()
                                _LOGGER.info("查询条件添加地址为:" + key)
                                _copy_url_handler.add_geo_scope(value)
                                urls_list.append(_copy_url_handler.get_my_url())
                        _copy_url_handler = deepcopy(url_handler)
                        for i in self.geo_list:
                            for key, value in i.items():
                                _copy_url_handler.add_bool_geo_scope(value)
                        urls_list.append(_copy_url_handler.get_my_url())
                        return urls_list

                    elif _type == 21:
                        _ready_index = kwargs["_ready_index"]
                        _copy_url_handler = deepcopy(url_handler)
                        _copy_url_handler.clear_geo_scope()
                        url_handler.add_geo_scope("tw:0")
                        for i in self.geo_list[1:_ready_index]:
                            for key, value in i.items():
                                _copy_url_handler.add_bool_geo_scope(value)
                        return [_copy_url_handler.get_my_url()]

                    elif _type == 3:
                        urls_list = []
                        for i in self.college_list:
                            for key, value in i.items():
                                _copy_url_handler = deepcopy(url_handler)
                                _copy_url_handler.clear_ed_area()
                                _LOGGER.info("查询条件大学为:" + key)
                                _copy_url_handler.add_ed(value)
                                urls_list.append(_copy_url_handler.get_my_url())
                        _copy_url_handler = deepcopy(url_handler)
                        for i in self.college_list:
                            for key, value in i.items():
                                _copy_url_handler.add_bool_ed(value)
                        urls_list.append(_copy_url_handler.get_my_url())
                        return urls_list

                    elif _type == 31:
                        _ready_index = kwargs["_ready_index"]
                        _copy_url_handler = deepcopy(url_handler)
                        _copy_url_handler.clear_ed_area()
                        for i in self.college_list[:_ready_index]:
                            for key, value in i.items():
                                _copy_url_handler.add_bool_ed(value)
                        return [_copy_url_handler.get_my_url()]

                    elif _type == 4:
                        urls_list = []
                        keywords = ["學生 OR 大學",
                                    "公司 OR 職員 OR 經理",
                                    "主任 OR 保险 OR 經紀",
                                    "運動 OR 建築",
                                    "藝術 OR 買",
                                    "文化 OR 附中",
                                    "附屬 OR 官員",
                                    "學院 OR 教育",
                                    "NOT (學生 OR 大學 OR 公司 OR 職員 OR 經理 OR 主任 OR 保险 OR 經紀)"
                                    ]
                        for i in random.choices(keywords, k=4):
                            _copy_url_handler = deepcopy(url_handler)
                            _copy_url_handler.add_keyword(i)
                            urls_list.append(_copy_url_handler.get_my_url())
                        return urls_list

                    return [url_handler.get_my_url()]

                def gen_data_getter_urls(my_historyid, my_seesionid, my_cachekey,
                                         my_linkcontext, pages):
                    if pages > 40:
                        pages = 40
                    return [
                        f"https://www.linkedin.com/recruiter/api/smartsearch?reset=PATL&reset=ECTL&reset=LRTL&reset=sortBy&searchHistoryId={my_historyid}&searchCacheKey={my_cachekey}&searchSessionId={my_seesionid}&linkContext={my_linkcontext}&doExplain=false&start={my_start*25}&origin=SRFS"
                        for my_start in range(pages)]

                async def clear_and_search(_url):
                    nonlocal totalcount
                    driver.find_element_by_xpath("//button[@data-tracking-control-name='clearSearchBtn']").click()
                    totalcount = totalcount + 1
                    await  asyncio.sleep(random.randint(10, 20))
                    driver.get(_url)
                    if totalcount > 2000:
                        driver.quit()
                        await  asyncio.sleep(random.randint(60 * 60, 90 * 60))
                        self.rebuild()
                        self.login(username, password)
                        totalcount = 1

                async def get_info():
                    _LOGGER.info("开始抓取个人info")
                    for sub_1_url in gen_search_url(gen_search_url(_, _type=0)[0], _type=1):
                        await clear_and_search(sub_1_url)
                        await asyncio.sleep(2)
                        total_count = check_result_count()
                        sub_result_count = []
                        if total_count > 1500:
                            name_and_geo_search_url_list = gen_search_url(sub_1_url, _type=2)
                            name_and_geo_search_url_list.reverse()
                            _index = 0
                            # 如果根据姓名和地址的组合
                            for _sub_1_sub_1_url in name_and_geo_search_url_list:
                                _LOGGER.info("当前访问的url" + _sub_1_sub_1_url)
                                print("当前访问的url" + _sub_1_sub_1_url)
                                await clear_and_search(_sub_1_sub_1_url)
                                await asyncio.sleep(random.randint(2, 5))
                                _count = check_result_count()
                                sub_result_count.append(_count)
                                # 如果根据姓名和地址的组合还是大于1000个人则使用，关键字和大学同在过滤一次
                                if _count > 1500:
                                    for _sub_14_sub_1_url in gen_search_url(_sub_1_sub_1_url, _type=4):
                                        await clear_and_search(_sub_14_sub_1_url)
                                        await asyncio.sleep(random.randint(2, 5))
                                        total_sub_14_count = check_result_count()
                                        if total_sub_14_count > 1500:
                                            sub_143_result_count = []
                                            for _sub_143_sub_1_url in gen_search_url(_sub_14_sub_1_url, _type=3):
                                                await clear_and_search(_sub_143_sub_1_url)
                                                await asyncio.sleep(random.randint(2, 5))
                                                _sub_143_count = check_result_count()
                                                sub_143_result_count.append(_sub_143_count)
                                                await get_person_info()
                                                if 0 < total_count - sum(sub_result_count) <= 1000:
                                                    await clear_and_search(gen_search_url(_sub_143_sub_1_url, _type=31, _ready_index=_index)[0])
                                                    await get_person_info()
                                                    break
                                        else:
                                            await get_person_info()
                                else:
                                    await get_person_info()
                                _index += 1

                                # 如果前几次的爬取的剩余总量少于1000，则直接用反向选择
                                if 0 < total_count - sum(sub_result_count) <= 1000:
                                    if _sub_1_sub_1_url == name_and_geo_search_url_list[-1]:
                                        break
                                    else:
                                        await clear_and_search(gen_search_url(_sub_1_sub_1_url, _type=21, _ready_index=_index)[0])
                                        await get_person_info()
                                        break
                        else:
                            await get_person_info()

                    await asyncio.sleep(3)

                async def get_person_info():
                    _LOGGER.debug("开始抓取个人详情")
                    await set_cookie()
                    total_count = check_result_count()
                    page_size = math.ceil(total_count / 25)
                    _url = furl(driver.current_url)
                    # print("page size" + str(page_size))

                    for data_fetch_url in gen_data_getter_urls(_url.args["searchHistoryId"], _url.args["searchSessionId"], _url.args["searchCacheKey"], _url.args["linkContext"], page_size):
                        # print(data_fetch_url)
                        _response = session.get(data_fetch_url)
                        self.client.linkedin_json.insert_one({"url": data_fetch_url, "result": _response.text})

                def check_result_count():
                    try:
                        p_count = driver.find_element_by_xpath("//div[@id='top-bar']/h2[@id]/span").text
                    except Exception as e:
                        p_count = 0
                    _LOGGER.info("当前页共有------" + str(p_count) + "------人")
                    if not p_count:
                        return 0
                    else:
                        return int(p_count.replace(",", ""))

                async def check_auth_status(_url):
                    if "consumer-email-challenge" in _url:
                        await self.check_and_relogin(driver, username, password)
                        await set_cookie()
                        return True
                    else:
                        return False

                print("开始抓取具体信息")
                await get_info()

    async def rule_xx(self):
        driver = self.driver
        driver.get("https://www.linkedin.com/cap/dashboard/home")
        driver.implicitly_wait(5)
        ele = driver.find_element_by_xpath("//div[@role='main']/div/h3")
        print(ele.text)
        try:
            friend_len = int(ele.text.split(" ")[0])
        except Exception as e:
            return

        def get_all_friend_list():
            _ = []
            while True:
                driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
                asyncio.sleep(10)
                _ = driver.find_elements_by_xpath("//div/ul/li[@class='mn-person-card connection-card']")
                if len(_) != friend_len:
                    driver.execute_script("return document.body.offsetHeight;")
                else:
                    break

            for element in _:
                href = element.find_element_by_xpath("//div/a").get
                profile = element.find_element_by_xpath("//div/a/img")

        def get_all_friend_list():
            while True:
                driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
                asyncio.sleep(10)
                if len(driver.find_elements_by_xpath("//div/ul/li[@class='mn-person-card connection-card']")) != friend_len:
                    driver.execute_script("return document.body.offsetHeight;")
                else:
                    break


class SearchNode(object):
    def __init__(self):
        self.parent_node: SearchNode = None
        self.child_nodes = []
        self.search_info = {}

    def save_or_update_search_info(self, info):
        self.search_info.update(info)

    def get_search_info(self, search_type=None, *args):
        _r = {}
        if search_type:
            for i in args:
                if i in self.search_info:
                    _r[i] = self.search_info[i]
        else:
            return self.search_info

    def add_child_node(self, node):
        if node in self.child_nodes:
            pass
        else:
            self.child_nodes.append(node)
            node.set_parent_node(self)

    def set_parent_node(self, node):
        if self.parent_node == node:
            pass
        else:
            self.parent_node = node
            node.add_child_node(self)


import json


class URLHandler(object):
    def __init__(self, _furl):
        self._furl: furl = furl(_furl)
        # self.gen_data_getter_url()

    def get_my_url(self):
        self._furl.args = list(set(self._furl.asdict()["query"]["params"]))
        self._furl.args.updateall()
        return self._furl.url

    def gen_data_getter_url(self):
        _furl = self._furl
        _furl.path.set("recruiter/api/smartsearch")
        _furl.args["origin"] = "PAGE"
        _furl.args.addlist("reset", ["PATL", "ECTL", "LRTL", "sortBy"])
        _furl.remove(["searchRequestId"])
        return self

    def gen_sub_search_url(self):
        _furl = self._furl
        _furl.path.set("recruiter/api/smartsearch")
        _furl.args["origin"] = "SRFS"
        _furl.args.addlist("reset", ["PATL", "ECTL", "LRTL", "sortBy"])
        _furl.remove(["searchRequestId"])
        return self

    def add_geo_scope(self, *geo_list):
        _LOGGER.info("混合如下geo地址" + json.dumps(geo_list))
        _furl = self._furl
        _furl.args.removevalues("reset", ["geoRegionScope", "G_C_R_SWR"])
        _furl.args.removevalues("geoRegionScope", ["LC"])
        _furl.args.addlist("reset", ["geoRegionScope", "G_C_R_SWR"])
        _furl.args.addlist("geoRegionScope", ["LC"])
        _furl.args.addlist("facet.G_C_R_SWR", geo_list)
        return self

    def add_bool_geo_scope(self, *geo_list):
        _LOGGER.info("bool混合如下geo地址" + json.dumps(geo_list))
        _furl = self._furl
        _furl.args.removevalues("reset", ["geoRegionScope", "G_C_R_SWR"])
        _furl.args.removevalues("geoRegionScope", ["LC"])
        _furl.args.addlist("reset", ["geoRegionScope", "G_C_R_SWR"])
        _furl.args.addlist("geoRegionScope", ["LC"])
        _furl.args.addlist("facet.G_C_R_SWR", [f"{i}_n" for i in geo_list])
        return self

    def clear_geo_scope(self):
        _LOGGER.info("清理geo地址")
        _furl = self._furl
        _furl.args.removevalues("reset", ["geoRegionScope", "G_C_R_SWR"])
        _furl.args.removevalues("geoRegionScope", ["LC"])
        _furl.remove("facet.G_C_R_SWR")
        return self

    def add_last_name(self, *last_name_list):
        _LOGGER.info("混合如下姓" + json.dumps(last_name_list))
        _furl = self._furl
        _furl.args.addlist("reset", ["lastName"])
        _furl.args.addlist("lastName", [" OR ".join(list(map((lambda x: f'"{x}"'), last_name_list)))])
        return self

    def clear_last_name(self):
        _LOGGER.info("清理所有姓")
        _furl = self._furl
        _furl.args.removevalues("reset", ["lastName"])
        _furl.remove("lastName")
        return self

    def add_ed(self, *ed_list):
        _furl = self._furl
        _furl.args.addlist("reset", ["ED_SWR"])
        _furl.args.addlist("facet.ED_SWR", ed_list)
        return self

    def add_bool_ed(self, *ed_list):
        _furl = self._furl
        _furl.args.addlist("reset", ["ED_SWR"])
        _furl.args.addlist("facet.ED_SWR", [f"{i}_n" for i in ed_list])
        return self

    def clear_ed_area(self):
        _furl = self._furl
        _furl.args.removevalues("reset", ["ED_SWR"])
        _furl.remove("facet.ED_SWR")
        return self

    def add_keyword(self, *keyword_list):
        _furl = self._furl
        _furl.args.addlist("reset", ["keywords"])
        _furl.args.addlist("keywords", [" OR ".join(list(map((lambda x: f'{x}'), keyword_list)))])
        return self

    def add_bool_keyword(self, *keyword_list):
        _furl = self._furl
        _furl.args.addlist("reset", ["keywords"])
        _furl.args.addlist("keywords", [f"{i}_n" for i in keyword_list])
        return self

    def clear_keyword_area(self):
        _furl = self._furl
        _furl.args.removevalues("reset", ["keywords"])
        _furl.remove("keywords")
        return self

    def add_graduated_year(self, start_year=None, end_year=None):
        _furl = self._furl
        _furl.args.addlist("reset", ["ED_YEAR"])
        _furl.args.addlist("facet.ED_YEAR", [f"{start_year}", f"{end_year}"])
        return self

    def clear_graduated_year(self, ):
        _furl = self._furl
        _furl.args.removevalues("reset", ["ED_YEAR"])
        _furl.remove("facet.ED_YEAR")
        return self


def custom_chrome_config(folder=None, is_headless=False, _proxy=None, ua=None):
    """

    :param folder:
        where the user data to put

        default is None
    :param is_headless:
        is use headless mode
    :param http_proxy:
        if use http_proxy
    :return:
        dict config
    """
    config = {}
    options = webdriver.ChromeOptions()
    desired_capabilities = DesiredCapabilities().CHROME.copy()
    options.add_argument('--headless') if is_headless else 1
    options.add_argument('--window-size=1800,30000')
    options.add_argument(f"--user-data-dir=.\\{folder}") if folder else 1
    options.add_argument(f"--disable-gpu")
    prefs = {
        # "profile.managed_default_content_settings.images": 2,
        "credentials_enable_service": False,
        "profile.default_content_setting_values.notifications": 2}
    options.add_experimental_option("prefs", prefs)
    print(_proxy)
    options.add_argument(f'--proxy-server=socks5://{_proxy}') if _proxy else 1
    options.add_argument("--disable-notifications")
    options.add_argument(f'--user-agent={ua}') if ua else 1
    config["chrome_options"] = options
    config["desired_capabilities"] = desired_capabilities
    return config
